
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
from lightgbm import LGBMClassifier
import optuna
from sklearn.metrics import f1_score, roc_auc_score
from sklearn.model_selection import train_test_split
from xgboost import XGBClassifier
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.feature_selection import mutual_info_classif
from sklearn.model_selection import StratifiedKFold, RandomizedSearchCV, cross_val_score, GridSearchCV
from sklearn.preprocessing import KBinsDiscretizer

warnings.filterwarnings('ignore')
pd.set_option('display.max_columns',None)


# to read the train.csv dataset into a dataframe 
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

# to see the overview of the table
train.head()


# to get the info of the table
train.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 12516 entries, 0 to 12515
Data columns (total 21 columns):
 #   Column                Non-Null Count  Dtype  
---  ------                --------------  -----  
 0   Entry_id              12516 non-null  int64  
 1   age                   12516 non-null  int64  
 2   pay_schedule          12516 non-null  object 
 3   home_owner            12516 non-null  int64  
 4   income                12516 non-null  int64  
 5   months_employed       12516 non-null  int64  
 6   years_employed        12516 non-null  int64  
 7   current_address_year  12516 non-null  int64  
 8   personal_account_m    12516 non-null  int64  
 9   personal_account_y    12516 non-null  int64  
 10  has_debt              12516 non-null  int64  
 11  amount_requested      12516 non-null  int64  
 12  risk_score            12516 non-null  int64  
 13  risk_score_2          12516 non-null  float64
 14  risk_score_3          12516 non-null  float64
 15  risk_score_4          12516 non-null  float64
 16  risk_score_5          12516 non-null  float64
 17  ext_quality_score     12516 non-null  float64
 18  ext_quality_score_2   12516 non-null  float64
 19  inquiries_last_month  12516 non-null  int64  
 20  e_signed              12516 non-null  int64  
dtypes: float64(6), int64(14), object(1)
memory usage: 2.0+ MB


# to get the shape of the table
train.shape

(12516, 21)


# to check for the presence of null values
train.isnull().sum()

Entry_id                0
age                     0
pay_schedule            0
home_owner              0
income                  0
months_employed         0
years_employed          0
current_address_year    0
personal_account_m      0
personal_account_y      0
has_debt                0
amount_requested        0
risk_score              0
risk_score_2            0
risk_score_3            0
risk_score_4            0
risk_score_5            0
ext_quality_score       0
ext_quality_score_2     0
inquiries_last_month    0
e_signed                0
dtype: int64


# to check for duplicate rows
sum(train.duplicated())

0


# to check for duplicated records in the entry_id column
train['Entry_id'].duplicated().sum()

6


# to print out the rows where the ID is in the IDs of duplicated rows
ids = train['Entry_id']
duplicate_id = ids[ids.duplicated()]
train[ids.isin(duplicate_id)].sort_values('Entry_id')


# to check for the percentage of unique records
train.nunique()/len(train)

Entry_id                0.999521
age                     0.005753
pay_schedule            0.000320
home_owner              0.000160
income                  0.155002
months_employed         0.000959
years_employed          0.001358
current_address_year    0.001039
personal_account_m      0.000959
personal_account_y      0.001198
has_debt                0.000160
amount_requested        0.007430
risk_score              0.109460
risk_score_2            0.983301
risk_score_3            0.283477
risk_score_4            0.989773
risk_score_5            0.985858
ext_quality_score       0.983381
ext_quality_score_2     0.983701
inquiries_last_month    0.002397
e_signed                0.000160
dtype: float64


# to check for the descriptive statistics of the numerical columns
train.describe()


# to visualize the distribution of the data

train.hist(figsize = (40, 60), color = 'turquoise');


# to count the number of e-signed clients 
sorted_counts = train['e_signed'].value_counts()
print(sorted_counts)

# to represent it in a pie plot
plt.figure(figsize = (10,6))
labels = ['E-signed', 'Not E-signed']
plt.pie(sorted_counts, labels = labels, radius = 1.8, autopct = '%0.2f%%', colors = ['turquoise', 'lightgray'],
        textprops = {'fontsize' : 12}, startangle= 90, counterclock= False, wedgeprops= {'width' : 1.0});
plt.axis('equal')
plt.title('Percentage of E-signed Clients', fontsize = 14 )
plt.legend(loc = 'upper right', fontsize = 12)
plt.show();

1    6766
0    5750
Name: e_signed, dtype: int64


# first lets check the number of clients in debt
debt_counts = train['has_debt'].value_counts()
print(debt_counts)

# to represent it in a pie plot
plt.figure(figsize = (10,6))
labels = ['Have Debt', 'No Debt']
plt.pie(debt_counts, labels = labels, radius = 1.8, autopct = '%0.2f%%', colors = ['turquoise', 'lightgray'],
        textprops = {'fontsize' : 12}, startangle= 90, counterclock= False, wedgeprops= {'width' : 1.0});
plt.axis('equal')
plt.title('Percentage of Clients who Have Debt', fontsize = 14 )
plt.legend(loc = 'upper right', fontsize = 12)
plt.show();

1    9956
0    2560
Name: has_debt, dtype: int64


# to check have_debt along with e-signed to explore relationship
plt.figure(figsize = [16,5])

# left plot 
# this is for clients who have debt
plt.subplot(1,2,1)
debt = train[train['has_debt'] == 1]

# to count the number of e-signed clients
debt_esign_counts = debt['e_signed'].value_counts()
print(debt_esign_counts)

# to represent it in a pie plot
labels = ['E-signed', 'Not E-signed']
plt.pie(debt_esign_counts, labels = labels, radius = 1.8, autopct = '%0.2f%%', colors = ['turquoise', 'lightgray'],
        textprops = {'fontsize' : 12}, startangle= 90, counterclock= False, wedgeprops= {'width' : 1.0});
plt.axis('equal')
plt.title('Percentage of E-signed Clients who has Debt', fontsize = 14 )
plt.legend(loc = 'upper right', fontsize = 12)

# right plot
# this is for clients who do not have debt
plt.subplot(1,2,2)
not_debt = train[train['has_debt'] == 0]

# to count the number of e-signed clients
not_debt_esign_counts = not_debt['e_signed'].value_counts()
print(not_debt_esign_counts)

# to represent it in a pie plot
labels = ['E-signed', 'Not E-signed']
plt.pie(not_debt_esign_counts, labels = labels, radius = 1.8, autopct = '%0.2f%%', colors = ['turquoise', 'lightgray'],
        textprops = {'fontsize' : 12}, startangle= 90, counterclock= False, wedgeprops= {'width' : 1.0});
plt.axis('equal')
plt.title('Percentage of E-signed Clients who has no Debt', fontsize = 14 )
plt.legend(loc = 'upper right', fontsize = 12);

1    5465
0    4491
Name: e_signed, dtype: int64
1    1301
0    1259
Name: e_signed, dtype: int64


# first lets check the number of clients in debt
house_owner = train['home_owner'].value_counts()
print(house_owner)

# to represent it in a pie plot
plt.figure(figsize = (10,6))
labels = ['Not Home Owner', 'Home Owner']
plt.pie(house_owner, labels = labels, radius = 1.8, autopct = '%0.2f%%', colors = ['lightgray', 'turquoise'],
        textprops = {'fontsize' : 12}, startangle= 90, counterclock= False, wedgeprops= {'width' : 1.0});
plt.axis('equal')
plt.title('Percentage of Clients who are Home Owners', fontsize = 14 )
plt.legend(loc = 'upper right', fontsize = 12)
plt.show();

0    7196
1    5320
Name: home_owner, dtype: int64


# to check House Owner along with e-signed to explore relationship
plt.figure(figsize = [16,5])

# left plot 
# this is for clients who have debt
plt.subplot(1,2,1)
home = train[train['home_owner'] == 1]

# to count the number of e-signed clients
home_esign_counts = home['e_signed'].value_counts()
print(home_esign_counts)

# to represent it in a pie plot
labels = ['E-signed', 'Not E-signed']
plt.pie(home_esign_counts, labels = labels, radius = 1.8, autopct = '%0.2f%%', colors = ['turquoise', 'lightgray'],
        textprops = {'fontsize' : 12}, startangle= 90, counterclock= False, wedgeprops= {'width' : 1.0});
plt.axis('equal')
plt.title('Percentage of E-signed Clients who are Home Owners', fontsize = 14 )
plt.legend(loc = 'upper right', fontsize = 12)

# right plot
# this is for clients who do not have debt
plt.subplot(1,2,2)
not_home = train[train['home_owner'] == 0]

# to count the number of e-signed clients
not_home_esign_counts = not_home['e_signed'].value_counts()
print(not_home_esign_counts)

# to represent it in a pie plot
labels = ['E-signed', 'Not E-signed']
plt.pie(not_home_esign_counts, labels = labels, radius = 1.8, autopct = '%0.2f%%', colors = ['turquoise', 'lightgray'],
        textprops = {'fontsize' : 12}, startangle= 90, counterclock= False, wedgeprops= {'width' : 1.0});
plt.axis('equal')
plt.title('Percentage of E-signed Clients who are not Home Owners', fontsize = 14 )
plt.legend(loc = 'upper right', fontsize = 12);

1    2707
0    2613
Name: e_signed, dtype: int64
1    4059
0    3137
Name: e_signed, dtype: int64


# first let us find the age distribution of the clients 
plt.figure(figsize= [20,10])
sns.set_style('white')

# left plot showing the histogram distribution of age
plt.subplot(1,2,1)
bins = np.arange(0, train['age'].max() + 5, 5)
sns.distplot(train['age'], kde = False, bins = bins, color= 'turquoise');
plt.title("Histogram Showing the Distribuion of Clients Age", fontsize= 16)
plt.yticks(fontsize = 14)
plt.xticks(fontsize = 14)
plt.xlabel("Age", fontsize = 14)

# right plot showing the kernel density plot of Client's Age
plt.subplot(1,2,2)
bins = np.arange(0, train['age'].max() + 5, 5)
sns.distplot(train['age'], hist = False, bins = bins, color= 'turquoise');
plt.title("Kernel Density Plot of CLients Age", fontsize= 16)
plt.yticks(fontsize = 14)
plt.xticks(fontsize = 14)
plt.xlabel("Age", fontsize = 14);


# let us observe the age distribution of the clients with respect to e-signed 
plt.figure(figsize= [20,10])
sns.set(style = 'dark')

# left plot showing the histogram distribution of age
plt.subplot(1,2,1)
esign = train[train['e_signed'] == 1]
bins = np.arange(0, esign['age'].max() + 5, 5)
sns.distplot(esign['age'], bins = bins, color= 'turquoise');
plt.title("Distribuion of Clients Age that are E-signed")

# right plot showing the kernel density plot of Client's Age
plt.subplot(1,2,2)
not_esign = train[train['e_signed'] == 0]
bins = np.arange(0, not_esign['age'].max() + 5, 5)
sns.distplot(not_esign['age'], bins = bins, color= 'turquoise');
plt.title("Distribuion of Clients Age that are not E-signed");


# observing the overview of the table
train.head()


# to plot a countplot 
sns.set_style('white')
counts = train['pay_schedule'].value_counts()
counts_order = counts.index
sns.countplot(data = train, x = 'pay_schedule', color= 'turquoise', order= counts_order);


# to examine the countplot based on E-signed 
counts = train['pay_schedule'].value_counts()
counts_order = counts.index
sns.catplot(data = train, x = 'pay_schedule', kind = 'count', color= 'turquoise', order= counts_order, col = 'e_signed');
plt.show();


# for clarity purpose, a heatmap is plotted to show the counts 
plt.figure(figsize= (12,8))
pay_counts = train.groupby(['pay_schedule','e_signed']).size()
pay_counts = pay_counts.reset_index(name= 'count')
pay_counts = pay_counts.pivot(columns = 'pay_schedule', index = 'e_signed', values= 'count')
sns.heatmap(pay_counts, annot= True, fmt= 'd', cmap= 'PuBuGn')
plt.yticks(fontsize = 14)
plt.xticks(fontsize = 14)
plt.ylabel('E_Signed', fontsize = 14)
plt.xlabel('Pay Schedule', fontsize = 14)
plt.title("HeatMap showing Count of Pay Schedules for clients for E_signed Loans", fontsize= 16);


train['amount_requested'].describe()

count    12516.000000
mean       950.251518
std        706.915146
min        350.000000
25%        600.000000
50%        700.000000
75%       1100.000000
max      10200.000000
Name: amount_requested, dtype: float64


# first let us find the distribution of the amount requested by clients 
plt.figure(figsize= [20,10])

# left plot showing the histogram distribution of amount requested by clients
plt.subplot(1,2,1)
bins = np.arange(0, train['amount_requested'].max() + 500, 500)
sns.distplot(train['amount_requested'], kde = False, bins = bins, color= 'turquoise');
plt.title("Histogram Showing the Distribuion of Amount Requested", fontsize= 16)
plt.yticks(fontsize = 14)
plt.xticks(fontsize = 14)
plt.xlabel("Amount Requested", fontsize = 14)

# right plot showing the kernel density plot of amount requested by clients
plt.subplot(1,2,2)
bins = np.arange(0, train['amount_requested'].max() + 500, 500)
sns.distplot(train['amount_requested'], hist = False, bins = bins, color= 'turquoise');
plt.title("Kernel Density Plot of Amount Requested", fontsize= 16)
plt.yticks(fontsize = 14)
plt.xticks(fontsize = 14)
plt.xlabel("Amount Requested", fontsize = 14);


# to examine the relationship between Amount requested and e-signed
plt.figure(figsize= (12,8))
sns.violinplot(data= train, x = 'e_signed', y= 'amount_requested', color= 'turquoise')
plt.yticks([0, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000, 11000], fontsize = 14)
plt.xticks(fontsize = 14)
plt.ylabel('Amount Requested', fontsize = 14)
plt.xlabel('E_signed', fontsize = 14)
plt.title("Distribution of Amount Requested with respect to E_signed", fontsize= 16);


# to plot a scatter plot income and amount requested
plt.figure(figsize= (12,8))
sns.regplot(x = 'income', y = 'amount_requested', data = train, color = 'turquoise', truncate= False)
plt.yticks(fontsize = 14)
plt.xticks(fontsize = 14)
plt.ylabel('Amount Requested', fontsize = 14)
plt.xlabel("Client's Income", fontsize = 14)
plt.title("Relationship Between Amount Requested and Client Income", fontsize= 16)
plt.show();


plt.figure(figsize= (12,8))
hue_colors = {0: 'lightgray', 1: 'turquoise'}
sns.scatterplot(x = 'income', y = 'amount_requested', hue= 'e_signed', data = train, palette= hue_colors)

# setting an amount requested mark at 4200
plt.axhline(4200, ls= '--', color= 'black')
plt.yticks(fontsize = 14)
plt.xticks(fontsize = 14)
plt.ylabel('Amount Requested', fontsize = 14)
plt.xlabel("Client's Income", fontsize = 14)
plt.title("Relationship Between Amount Requested and Client Income", fontsize= 16)
plt.show();


# to seperate the scatterplot based on e-signed
plt.figure(figsize=[20, 12])
g = sns.relplot(x = 'income', y = 'amount_requested', data = train, kind = 'scatter', col = 'e_signed', color = 'turquoise')
axes = g.axes.flatten()

for i, ax in enumerate(axes):
    ax.axhline(4200, ls = '--', c = 'black')
plt.show();

<Figure size 1440x864 with 0 Axes>


# to seperate the scatterplot based on e-signed and has debt

g = sns.relplot(x = 'income', y = 'amount_requested', data = train, kind = 'scatter', col = 'e_signed', row = 'has_debt',
            color = 'turquoise')
axes = g.axes.flatten()

for i, ax in enumerate(axes):
    ax.axhline(4200, ls = '--', c = 'black')

plt.show();


# to calculate the mean external quality score
train['mean_quality'] = train[['ext_quality_score', 'ext_quality_score_2']].mean(axis = 1)

# to calculate the mean risk score using risk score 2, 3, 4 & 5
col = train.loc[:, "risk_score_2":"risk_score_5"]
train['mean_risk_score'] = col.mean(axis= 1)

# to check if the columns have been added
train.head();


# to check the relationship between mean_risk_score, ext_quality_score and income
column = train[['mean_risk_score', 'mean_quality', 'income']]
print(column.head())

# using pairplot to observe relationships
sns.pairplot(column, height= 4);

   mean_risk_score  mean_quality  income
0         0.661151      0.480918    3135
1         0.688483      0.744634    3590
2         0.649061      0.542556    2303
3         0.744522      0.634130    2795
4         0.652702      0.625905    5000


# to check the correlation between mean quality score and mean risk score
train['mean_risk_score'].corr(train['mean_quality'])

0.4190188215401355


sns.relplot(data= train, x = 'mean_risk_score', y = 'mean_quality', col= 'e_signed', kind= 'scatter', color = 'turquoise')
plt.show();


# to check the correlation
esign = train[train['e_signed'] == 1]
not_esign = train[train['e_signed'] == 0]

esign_cor = esign['mean_risk_score'].corr(esign['mean_quality'])
not_esign_cor = not_esign['mean_risk_score'].corr(not_esign['mean_quality'])

print(esign_cor)
print(not_esign_cor)

0.4270416332733701
0.40958524675311725


# first let us find the distribution of the Number of Inquiries Last Month
plt.figure(figsize= [20,10])

# left plot showing the histogram distribution of Number of Inquiries Last Month
plt.subplot(1,2,1)
bins = np.arange(0, train['inquiries_last_month'].max() + 2, 2)
sns.distplot(train['inquiries_last_month'], kde = False, bins = bins, color= 'turquoise');
plt.title("Histogram Showing the Distribuion of the Number of Inquiries Last Month", fontsize= 16)
plt.yticks(fontsize = 14)
plt.xticks(fontsize = 14)
plt.xlabel("inquiries_last_month", fontsize = 14)

# right plot showing the kernel density plot of Number of Inquiries Last Month
plt.subplot(1,2,2)
bins = np.arange(0, train['inquiries_last_month'].max() + 4, 4)
sns.distplot(train['inquiries_last_month'], hist = False, bins = bins, color= 'turquoise');
plt.title("Kernel Density Plot of the Number of Inquiries Last Month", fontsize= 16)
plt.yticks(fontsize = 14)
plt.xticks(fontsize = 14)
plt.xlabel("inquiries_last_month", fontsize = 14);


# to examine the relationship between Amount requested and e-signed
plt.figure(figsize= (12,8))
sns.violinplot(data= train, x = 'e_signed', y= 'inquiries_last_month', color= 'turquoise')
plt.yticks(fontsize = 14)
plt.xticks(fontsize = 14)
plt.ylabel('Number of Inquiries last Month', fontsize = 14)
plt.xlabel('E_signed', fontsize = 14)
plt.title("Distribution of Number of Inquiries last Month with respect to E_signed", fontsize= 16);


# to examine the relationship further using has_debt variable
plt.figure(figsize= [20,25])

debts = train[train['has_debt'] == 1]
nodebts = train[train['has_debt'] == 0]

# left plot is for clients who have debt
plt.subplot(2,1,1)
sns.violinplot(data= debts, x = 'e_signed', y= 'inquiries_last_month', color= 'turquoise')
plt.yticks(fontsize = 14)
plt.xticks(fontsize = 14)
plt.ylabel('Number of Inquiries last Month', fontsize = 14)
plt.xlabel('E_signed', fontsize = 14)
plt.title("Distribution of Number of Inquiries last Month For Clients with Debt with respect to E_signed", fontsize= 16);

# right plot is for clients who do not have debt
plt.subplot(2,1,2)
sns.violinplot(data= nodebts, x = 'e_signed', y= 'inquiries_last_month', color= 'turquoise')
plt.yticks(fontsize = 14)
plt.xticks(fontsize = 14)
plt.ylabel('Number of Inquiries last Month', fontsize = 14)
plt.xlabel('E_signed', fontsize = 14)
plt.title("Distribution of Number of Inquiries last Month For Clients without Debt with respect to E_signed", fontsize= 16);


train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')


# computing 'class_weight'
from sklearn.utils import class_weight
class_weight = dict(zip(np.unique(train['e_signed']), class_weight.compute_class_weight(class_weight = 'balanced',classes = np.unique(train['e_signed']),
                                                                                        y = train['e_signed']))) 
class_weight

{0: 1.0883478260869566, 1: 0.9249187112030742}


#stratified kfold validation scheme:
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state = 752)


# checking for correlations that can be explored for feature engineering;
plt.figure(figsize=(15,8))
sns.heatmap(train.iloc[:,1:train.shape[1]-1].corr(),annot=True, cmap='viridis');


X = train.drop(['Entry_id','e_signed'], axis=1)
y = train.e_signed
# -----------------------------------------------------------------
X_pay_schedule = pd.get_dummies(X['pay_schedule']) #dummy variables;
X_copy = pd.concat((X,X_pay_schedule), axis=1)
X_copy.drop('pay_schedule',axis=1,inplace=True)
X = X_copy.copy()

# on test_dataset:
test_entry_Id = test['Entry_id']
test_pay_schedule = pd.get_dummies(test['pay_schedule'])
test_copy = pd.concat((test,test_pay_schedule), axis=1)
test_copy.drop('pay_schedule',axis=1,inplace=True)
test_copy.drop('Entry_id',axis=1,inplace=True)


'''discretizer = KBinsDiscretizer(n_bins=5, encode='ordinal', strategy='quantile')

X['income_bins'] = discretizer.fit_transform(X['income'].values.reshape(-1,1).astype('float'))

X['amount_requested_bin'] = discretizer.fit_transform(X['amount_requested'].values.reshape(-1,1).astype('float'))

X['mean_risk_score'] = X[['risk_score_2','risk_score_3','risk_score_4',	'risk_score_5']].mean(axis=1)

X['mean_ext_quality'] = X[['ext_quality_score','ext_quality_score_2']].mean(axis=1)'''

"discretizer = KBinsDiscretizer(n_bins=5, encode='ordinal', strategy='quantile')\n\nX['income_bins'] = discretizer.fit_transform(X['income'].values.reshape(-1,1).astype('float'))\n\nX['amount_requested_bin'] = discretizer.fit_transform(X['amount_requested'].values.reshape(-1,1).astype('float'))\n\nX['mean_risk_score'] = X[['risk_score_2','risk_score_3','risk_score_4',\t'risk_score_5']].mean(axis=1)\n\nX['mean_ext_quality'] = X[['ext_quality_score','ext_quality_score_2']].mean(axis=1)"


X.head()


# in other to capture non-linear relationships, we compute the mutual_information_score of the features;
def make_mi_scores(X, y):
    X = X.copy()
    for colname in X.select_dtypes(["object", "category"]):
        X[colname], _ = X[colname].factorize()
    # All discrete features should now have integer dtypes
    discrete_features = [pd.api.types.is_integer_dtype(t) for t in X.dtypes]
    mi_scores = mutual_info_classif(X, y, discrete_features=discrete_features, random_state=0)
    mi_scores = pd.Series(mi_scores, name="MI Scores", index=X.columns)
    mi_scores = mi_scores.sort_values(ascending=False)
    return mi_scores


# to plot the score of Mutual Information 
def plot_mi_scores(scores):
    scores = scores.sort_values(ascending=True)
    width = np.arange(len(scores))
    ticks = list(scores.index)
    plt.barh(width, scores)
    plt.yticks(width, ticks)
    plt.title("Mutual Information Scores")


mi_scores = make_mi_scores(X,y)

plt.figure(figsize=(12,8))
plot_mi_scores(mi_scores)


lgb_clf = LGBMClassifier(random_state=752, class_weight=class_weight)


# Carrying Hyper Parameter Optuna Optimization using a Bayesian Optimizer;
def objective(trial):
    params ={
    'max_bin': trial.suggest_int('max_bin', 30, 150),
    'max_depth': trial.suggest_int('max_depth', 4, 15),
    'subsample': trial.suggest_uniform('subsample', 0.6,  1.0),
    'min_child_samples': trial.suggest_int('min_child_samples', 30, 100),
    'num_leaves': trial.suggest_int('num_leaves', 30, 100),
    'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.6, 0.9),
    'learning_rate': trial.suggest_loguniform('learning_rate', 0.1,  0.4),
    'bagging_freq': trial.suggest_int('bagging_freq', 6, 10),
    'lambda_l1': trial.suggest_uniform('lambda_l1', 0.0, 10.0),
    'lambda_l2': trial.suggest_uniform('lambda_l2', 0.0, 10.0),
    }
    lgb_clf.set_params(**params)

    
    
    return np.mean(cross_val_score(lgb_clf, X, y, cv=skf, scoring='roc_auc'))


study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=15)

[I 2022-10-18 11:35:23,321] A new study created in memory with name: no-name-6d772233-9ca0-488e-a024-3198ee965290

[LightGBM] [Warning] lambda_l1 is set=3.660429140289728, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.660429140289728
[LightGBM] [Warning] lambda_l2 is set=3.6702986906413804, reg_lambda=0.0 will be ignored. Current value: lambda_l2=3.6702986906413804
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=3.660429140289728, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.660429140289728
[LightGBM] [Warning] lambda_l2 is set=3.6702986906413804, reg_lambda=0.0 will be ignored. Current value: lambda_l2=3.6702986906413804
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=3.660429140289728, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.660429140289728
[LightGBM] [Warning] lambda_l2 is set=3.6702986906413804, reg_lambda=0.0 will be ignored. Current value: lambda_l2=3.6702986906413804
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=3.660429140289728, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.660429140289728
[LightGBM] [Warning] lambda_l2 is set=3.6702986906413804, reg_lambda=0.0 will be ignored. Current value: lambda_l2=3.6702986906413804
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=3.660429140289728, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.660429140289728
[LightGBM] [Warning] lambda_l2 is set=3.6702986906413804, reg_lambda=0.0 will be ignored. Current value: lambda_l2=3.6702986906413804
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=3.660429140289728, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.660429140289728
[LightGBM] [Warning] lambda_l2 is set=3.6702986906413804, reg_lambda=0.0 will be ignored. Current value: lambda_l2=3.6702986906413804
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=3.660429140289728, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.660429140289728
[LightGBM] [Warning] lambda_l2 is set=3.6702986906413804, reg_lambda=0.0 will be ignored. Current value: lambda_l2=3.6702986906413804
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=3.660429140289728, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.660429140289728
[LightGBM] [Warning] lambda_l2 is set=3.6702986906413804, reg_lambda=0.0 will be ignored. Current value: lambda_l2=3.6702986906413804
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=3.660429140289728, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.660429140289728
[LightGBM] [Warning] lambda_l2 is set=3.6702986906413804, reg_lambda=0.0 will be ignored. Current value: lambda_l2=3.6702986906413804
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=3.660429140289728, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.660429140289728
[LightGBM] [Warning] lambda_l2 is set=3.6702986906413804, reg_lambda=0.0 will be ignored. Current value: lambda_l2=3.6702986906413804
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10

[I 2022-10-18 11:35:35,354] Trial 0 finished with value: 0.6923767913269205 and parameters: {'max_bin': 41, 'max_depth': 10, 'subsample': 0.9867003879887545, 'min_child_samples': 64, 'num_leaves': 52, 'colsample_bytree': 0.7744347699848178, 'learning_rate': 0.1397854486575055, 'bagging_freq': 10, 'lambda_l1': 3.660429140289728, 'lambda_l2': 3.6702986906413804}. Best is trial 0 with value: 0.6923767913269205.

[LightGBM] [Warning] lambda_l1 is set=5.630950445534348, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.630950445534348
[LightGBM] [Warning] lambda_l2 is set=9.573659118627061, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.573659118627061
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=5.630950445534348, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.630950445534348
[LightGBM] [Warning] lambda_l2 is set=9.573659118627061, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.573659118627061
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=5.630950445534348, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.630950445534348
[LightGBM] [Warning] lambda_l2 is set=9.573659118627061, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.573659118627061
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=5.630950445534348, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.630950445534348
[LightGBM] [Warning] lambda_l2 is set=9.573659118627061, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.573659118627061
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=5.630950445534348, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.630950445534348
[LightGBM] [Warning] lambda_l2 is set=9.573659118627061, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.573659118627061
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=5.630950445534348, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.630950445534348
[LightGBM] [Warning] lambda_l2 is set=9.573659118627061, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.573659118627061
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=5.630950445534348, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.630950445534348
[LightGBM] [Warning] lambda_l2 is set=9.573659118627061, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.573659118627061
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=5.630950445534348, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.630950445534348
[LightGBM] [Warning] lambda_l2 is set=9.573659118627061, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.573659118627061
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=5.630950445534348, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.630950445534348
[LightGBM] [Warning] lambda_l2 is set=9.573659118627061, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.573659118627061
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=5.630950445534348, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.630950445534348
[LightGBM] [Warning] lambda_l2 is set=9.573659118627061, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.573659118627061
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7

[I 2022-10-18 11:35:42,647] Trial 1 finished with value: 0.6920266464095179 and parameters: {'max_bin': 64, 'max_depth': 12, 'subsample': 0.6240668724820215, 'min_child_samples': 32, 'num_leaves': 36, 'colsample_bytree': 0.6823320478190402, 'learning_rate': 0.11404079029070653, 'bagging_freq': 7, 'lambda_l1': 5.630950445534348, 'lambda_l2': 9.573659118627061}. Best is trial 0 with value: 0.6923767913269205.

[LightGBM] [Warning] lambda_l1 is set=9.830608328352218, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.830608328352218
[LightGBM] [Warning] lambda_l2 is set=6.0262929039209725, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.0262929039209725
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=9.830608328352218, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.830608328352218
[LightGBM] [Warning] lambda_l2 is set=6.0262929039209725, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.0262929039209725
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=9.830608328352218, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.830608328352218
[LightGBM] [Warning] lambda_l2 is set=6.0262929039209725, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.0262929039209725
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=9.830608328352218, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.830608328352218
[LightGBM] [Warning] lambda_l2 is set=6.0262929039209725, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.0262929039209725
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=9.830608328352218, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.830608328352218
[LightGBM] [Warning] lambda_l2 is set=6.0262929039209725, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.0262929039209725
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=9.830608328352218, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.830608328352218
[LightGBM] [Warning] lambda_l2 is set=6.0262929039209725, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.0262929039209725
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=9.830608328352218, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.830608328352218
[LightGBM] [Warning] lambda_l2 is set=6.0262929039209725, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.0262929039209725
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=9.830608328352218, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.830608328352218
[LightGBM] [Warning] lambda_l2 is set=6.0262929039209725, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.0262929039209725
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=9.830608328352218, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.830608328352218
[LightGBM] [Warning] lambda_l2 is set=6.0262929039209725, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.0262929039209725
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7
[LightGBM] [Warning] lambda_l1 is set=9.830608328352218, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.830608328352218
[LightGBM] [Warning] lambda_l2 is set=6.0262929039209725, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.0262929039209725
[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7

[I 2022-10-18 11:35:48,176] Trial 2 finished with value: 0.6788635560948342 and parameters: {'max_bin': 52, 'max_depth': 8, 'subsample': 0.644272973827263, 'min_child_samples': 35, 'num_leaves': 38, 'colsample_bytree': 0.7317253085581932, 'learning_rate': 0.23365141139363046, 'bagging_freq': 7, 'lambda_l1': 9.830608328352218, 'lambda_l2': 6.0262929039209725}. Best is trial 0 with value: 0.6923767913269205.

[LightGBM] [Warning] lambda_l1 is set=5.245279854663974, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.245279854663974
[LightGBM] [Warning] lambda_l2 is set=0.5023527141889284, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.5023527141889284
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=5.245279854663974, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.245279854663974
[LightGBM] [Warning] lambda_l2 is set=0.5023527141889284, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.5023527141889284
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=5.245279854663974, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.245279854663974
[LightGBM] [Warning] lambda_l2 is set=0.5023527141889284, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.5023527141889284
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=5.245279854663974, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.245279854663974
[LightGBM] [Warning] lambda_l2 is set=0.5023527141889284, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.5023527141889284
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=5.245279854663974, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.245279854663974
[LightGBM] [Warning] lambda_l2 is set=0.5023527141889284, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.5023527141889284
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=5.245279854663974, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.245279854663974
[LightGBM] [Warning] lambda_l2 is set=0.5023527141889284, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.5023527141889284
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=5.245279854663974, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.245279854663974
[LightGBM] [Warning] lambda_l2 is set=0.5023527141889284, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.5023527141889284
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=5.245279854663974, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.245279854663974
[LightGBM] [Warning] lambda_l2 is set=0.5023527141889284, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.5023527141889284
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=5.245279854663974, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.245279854663974
[LightGBM] [Warning] lambda_l2 is set=0.5023527141889284, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.5023527141889284
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=5.245279854663974, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.245279854663974
[LightGBM] [Warning] lambda_l2 is set=0.5023527141889284, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.5023527141889284
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10

[I 2022-10-18 11:35:55,396] Trial 3 finished with value: 0.6645654913796282 and parameters: {'max_bin': 47, 'max_depth': 6, 'subsample': 0.6694009075830384, 'min_child_samples': 65, 'num_leaves': 58, 'colsample_bytree': 0.8391950621383191, 'learning_rate': 0.3705937101294885, 'bagging_freq': 10, 'lambda_l1': 5.245279854663974, 'lambda_l2': 0.5023527141889284}. Best is trial 0 with value: 0.6923767913269205.

[LightGBM] [Warning] lambda_l1 is set=1.0597424860446036, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.0597424860446036
[LightGBM] [Warning] lambda_l2 is set=6.124718037571873, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.124718037571873
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=1.0597424860446036, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.0597424860446036
[LightGBM] [Warning] lambda_l2 is set=6.124718037571873, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.124718037571873
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=1.0597424860446036, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.0597424860446036
[LightGBM] [Warning] lambda_l2 is set=6.124718037571873, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.124718037571873
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=1.0597424860446036, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.0597424860446036
[LightGBM] [Warning] lambda_l2 is set=6.124718037571873, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.124718037571873
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=1.0597424860446036, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.0597424860446036
[LightGBM] [Warning] lambda_l2 is set=6.124718037571873, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.124718037571873
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=1.0597424860446036, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.0597424860446036
[LightGBM] [Warning] lambda_l2 is set=6.124718037571873, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.124718037571873
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=1.0597424860446036, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.0597424860446036
[LightGBM] [Warning] lambda_l2 is set=6.124718037571873, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.124718037571873
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=1.0597424860446036, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.0597424860446036
[LightGBM] [Warning] lambda_l2 is set=6.124718037571873, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.124718037571873
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=1.0597424860446036, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.0597424860446036
[LightGBM] [Warning] lambda_l2 is set=6.124718037571873, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.124718037571873
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=1.0597424860446036, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.0597424860446036
[LightGBM] [Warning] lambda_l2 is set=6.124718037571873, reg_lambda=0.0 will be ignored. Current value: lambda_l2=6.124718037571873
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10


study.best_value

0.6983593962984596


lgb_clf.set_params(**study.best_params)
lgb_clf
# Parameters and Hyperparameters are taken note of for later:

LGBMClassifier(bagging_freq=8,
               class_weight={0: 1.0883478260869566, 1: 0.9249187112030742},
               colsample_bytree=0.6569151459124567, lambda_l1=4.160651586352121,
               lambda_l2=7.732030318358097, learning_rate=0.10447652039740438,
               max_bin=83, max_depth=13, min_child_samples=51, random_state=752,
               subsample=0.8971640076320018)

LGBMClassifier(bagging_freq=8,
               class_weight={0: 1.0883478260869566, 1: 0.9249187112030742},
               colsample_bytree=0.6569151459124567, lambda_l1=4.160651586352121,
               lambda_l2=7.732030318358097, learning_rate=0.10447652039740438,
               max_bin=83, max_depth=13, min_child_samples=51, random_state=752,
               subsample=0.8971640076320018)


from optuna import Trial, visualization


optuna.visualization.plot_optimization_history(study)


optuna.visualization.plot_slice(study)


# resetting Parameter range in other to improve lgbm score:
def objective(trial):
    params ={
    'max_bin': trial.suggest_int('max_bin', 45, 50),
    'max_depth': trial.suggest_int('max_depth', 6, 7),
    'subsample': trial.suggest_uniform('subsample', 0.79,0.81 ),
    'min_child_samples': trial.suggest_int('min_child_samples', 71, 75),
    'num_leaves': trial.suggest_int('num_leaves', 91, 93),
    'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.85, 0.9),
    'learning_rate': trial.suggest_loguniform('learning_rate', 0.15,  0.25),
    'bagging_freq': trial.suggest_int('bagging_freq', 9, 10),
    'lambda_l1': trial.suggest_uniform('lambda_l1', 2.8, 2.9),
    'lambda_l2': trial.suggest_uniform('lambda_l2', 8.15, 8.30),
    }
    lgb_clf.set_params(**params)

    
    
    return np.mean(cross_val_score(lgb_clf, X, y, cv=skf, scoring='roc_auc'))


study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=20)

[I 2022-10-18 11:37:33,150] A new study created in memory with name: no-name-449440b7-c40f-41fc-8a64-7559ffbcbe23

[LightGBM] [Warning] lambda_l1 is set=2.8199376144013204, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8199376144013204
[LightGBM] [Warning] lambda_l2 is set=8.23193978054748, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.23193978054748
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8199376144013204, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8199376144013204
[LightGBM] [Warning] lambda_l2 is set=8.23193978054748, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.23193978054748
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8199376144013204, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8199376144013204
[LightGBM] [Warning] lambda_l2 is set=8.23193978054748, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.23193978054748
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8199376144013204, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8199376144013204
[LightGBM] [Warning] lambda_l2 is set=8.23193978054748, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.23193978054748
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8199376144013204, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8199376144013204
[LightGBM] [Warning] lambda_l2 is set=8.23193978054748, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.23193978054748
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8199376144013204, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8199376144013204
[LightGBM] [Warning] lambda_l2 is set=8.23193978054748, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.23193978054748
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8199376144013204, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8199376144013204
[LightGBM] [Warning] lambda_l2 is set=8.23193978054748, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.23193978054748
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8199376144013204, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8199376144013204
[LightGBM] [Warning] lambda_l2 is set=8.23193978054748, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.23193978054748
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8199376144013204, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8199376144013204
[LightGBM] [Warning] lambda_l2 is set=8.23193978054748, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.23193978054748
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8199376144013204, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8199376144013204
[LightGBM] [Warning] lambda_l2 is set=8.23193978054748, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.23193978054748
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10

[I 2022-10-18 11:37:42,277] Trial 0 finished with value: 0.6880573524823685 and parameters: {'max_bin': 48, 'max_depth': 7, 'subsample': 0.80746623030307, 'min_child_samples': 73, 'num_leaves': 92, 'colsample_bytree': 0.8764180684577638, 'learning_rate': 0.15565391169843945, 'bagging_freq': 10, 'lambda_l1': 2.8199376144013204, 'lambda_l2': 8.23193978054748}. Best is trial 0 with value: 0.6880573524823685.

[LightGBM] [Warning] lambda_l1 is set=2.8192130362366745, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8192130362366745
[LightGBM] [Warning] lambda_l2 is set=8.242163283033076, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.242163283033076
[LightGBM] [Warning] bagging_freq is set=9, subsample_freq=0 will be ignored. Current value: bagging_freq=9
[LightGBM] [Warning] lambda_l1 is set=2.8192130362366745, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8192130362366745
[LightGBM] [Warning] lambda_l2 is set=8.242163283033076, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.242163283033076
[LightGBM] [Warning] bagging_freq is set=9, subsample_freq=0 will be ignored. Current value: bagging_freq=9
[LightGBM] [Warning] lambda_l1 is set=2.8192130362366745, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8192130362366745
[LightGBM] [Warning] lambda_l2 is set=8.242163283033076, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.242163283033076
[LightGBM] [Warning] bagging_freq is set=9, subsample_freq=0 will be ignored. Current value: bagging_freq=9
[LightGBM] [Warning] lambda_l1 is set=2.8192130362366745, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8192130362366745
[LightGBM] [Warning] lambda_l2 is set=8.242163283033076, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.242163283033076
[LightGBM] [Warning] bagging_freq is set=9, subsample_freq=0 will be ignored. Current value: bagging_freq=9
[LightGBM] [Warning] lambda_l1 is set=2.8192130362366745, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8192130362366745
[LightGBM] [Warning] lambda_l2 is set=8.242163283033076, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.242163283033076
[LightGBM] [Warning] bagging_freq is set=9, subsample_freq=0 will be ignored. Current value: bagging_freq=9
[LightGBM] [Warning] lambda_l1 is set=2.8192130362366745, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8192130362366745
[LightGBM] [Warning] lambda_l2 is set=8.242163283033076, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.242163283033076
[LightGBM] [Warning] bagging_freq is set=9, subsample_freq=0 will be ignored. Current value: bagging_freq=9
[LightGBM] [Warning] lambda_l1 is set=2.8192130362366745, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8192130362366745
[LightGBM] [Warning] lambda_l2 is set=8.242163283033076, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.242163283033076
[LightGBM] [Warning] bagging_freq is set=9, subsample_freq=0 will be ignored. Current value: bagging_freq=9
[LightGBM] [Warning] lambda_l1 is set=2.8192130362366745, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8192130362366745
[LightGBM] [Warning] lambda_l2 is set=8.242163283033076, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.242163283033076
[LightGBM] [Warning] bagging_freq is set=9, subsample_freq=0 will be ignored. Current value: bagging_freq=9
[LightGBM] [Warning] lambda_l1 is set=2.8192130362366745, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8192130362366745
[LightGBM] [Warning] lambda_l2 is set=8.242163283033076, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.242163283033076
[LightGBM] [Warning] bagging_freq is set=9, subsample_freq=0 will be ignored. Current value: bagging_freq=9
[LightGBM] [Warning] lambda_l1 is set=2.8192130362366745, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8192130362366745
[LightGBM] [Warning] lambda_l2 is set=8.242163283033076, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.242163283033076
[LightGBM] [Warning] bagging_freq is set=9, subsample_freq=0 will be ignored. Current value: bagging_freq=9

[I 2022-10-18 11:37:48,577] Trial 1 finished with value: 0.6852029554257859 and parameters: {'max_bin': 46, 'max_depth': 7, 'subsample': 0.7938988996922094, 'min_child_samples': 75, 'num_leaves': 92, 'colsample_bytree': 0.8553452921516038, 'learning_rate': 0.17230960921114752, 'bagging_freq': 9, 'lambda_l1': 2.8192130362366745, 'lambda_l2': 8.242163283033076}. Best is trial 0 with value: 0.6880573524823685.

[LightGBM] [Warning] lambda_l1 is set=2.8309181961284193, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8309181961284193
[LightGBM] [Warning] lambda_l2 is set=8.266196069847373, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.266196069847373
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8309181961284193, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8309181961284193
[LightGBM] [Warning] lambda_l2 is set=8.266196069847373, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.266196069847373
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8309181961284193, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8309181961284193
[LightGBM] [Warning] lambda_l2 is set=8.266196069847373, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.266196069847373
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8309181961284193, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8309181961284193
[LightGBM] [Warning] lambda_l2 is set=8.266196069847373, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.266196069847373
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8309181961284193, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8309181961284193
[LightGBM] [Warning] lambda_l2 is set=8.266196069847373, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.266196069847373
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8309181961284193, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8309181961284193
[LightGBM] [Warning] lambda_l2 is set=8.266196069847373, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.266196069847373
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8309181961284193, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8309181961284193
[LightGBM] [Warning] lambda_l2 is set=8.266196069847373, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.266196069847373
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8309181961284193, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8309181961284193
[LightGBM] [Warning] lambda_l2 is set=8.266196069847373, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.266196069847373
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8309181961284193, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8309181961284193
[LightGBM] [Warning] lambda_l2 is set=8.266196069847373, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.266196069847373
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8309181961284193, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8309181961284193
[LightGBM] [Warning] lambda_l2 is set=8.266196069847373, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.266196069847373
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10

[I 2022-10-18 11:37:54,515] Trial 2 finished with value: 0.677312305267834 and parameters: {'max_bin': 47, 'max_depth': 7, 'subsample': 0.801796371124881, 'min_child_samples': 75, 'num_leaves': 92, 'colsample_bytree': 0.869616386185692, 'learning_rate': 0.24697537120185567, 'bagging_freq': 10, 'lambda_l1': 2.8309181961284193, 'lambda_l2': 8.266196069847373}. Best is trial 0 with value: 0.6880573524823685.

[LightGBM] [Warning] lambda_l1 is set=2.884140511514055, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.884140511514055
[LightGBM] [Warning] lambda_l2 is set=8.271358055675366, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.271358055675366
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.884140511514055, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.884140511514055
[LightGBM] [Warning] lambda_l2 is set=8.271358055675366, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.271358055675366
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.884140511514055, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.884140511514055
[LightGBM] [Warning] lambda_l2 is set=8.271358055675366, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.271358055675366
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.884140511514055, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.884140511514055
[LightGBM] [Warning] lambda_l2 is set=8.271358055675366, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.271358055675366
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.884140511514055, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.884140511514055
[LightGBM] [Warning] lambda_l2 is set=8.271358055675366, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.271358055675366
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.884140511514055, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.884140511514055
[LightGBM] [Warning] lambda_l2 is set=8.271358055675366, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.271358055675366
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.884140511514055, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.884140511514055
[LightGBM] [Warning] lambda_l2 is set=8.271358055675366, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.271358055675366
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.884140511514055, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.884140511514055
[LightGBM] [Warning] lambda_l2 is set=8.271358055675366, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.271358055675366
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.884140511514055, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.884140511514055
[LightGBM] [Warning] lambda_l2 is set=8.271358055675366, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.271358055675366
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.884140511514055, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.884140511514055
[LightGBM] [Warning] lambda_l2 is set=8.271358055675366, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.271358055675366
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10

[I 2022-10-18 11:38:03,174] Trial 3 finished with value: 0.6849672639054775 and parameters: {'max_bin': 48, 'max_depth': 6, 'subsample': 0.8065897390776072, 'min_child_samples': 71, 'num_leaves': 92, 'colsample_bytree': 0.8853656213372506, 'learning_rate': 0.22148377024769697, 'bagging_freq': 10, 'lambda_l1': 2.884140511514055, 'lambda_l2': 8.271358055675366}. Best is trial 0 with value: 0.6880573524823685.

[LightGBM] [Warning] lambda_l1 is set=2.8492144645553923, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8492144645553923
[LightGBM] [Warning] lambda_l2 is set=8.165710252546063, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.165710252546063
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8492144645553923, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8492144645553923
[LightGBM] [Warning] lambda_l2 is set=8.165710252546063, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.165710252546063
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8492144645553923, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8492144645553923
[LightGBM] [Warning] lambda_l2 is set=8.165710252546063, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.165710252546063
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8492144645553923, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8492144645553923
[LightGBM] [Warning] lambda_l2 is set=8.165710252546063, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.165710252546063
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8492144645553923, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8492144645553923
[LightGBM] [Warning] lambda_l2 is set=8.165710252546063, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.165710252546063
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8492144645553923, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8492144645553923
[LightGBM] [Warning] lambda_l2 is set=8.165710252546063, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.165710252546063
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8492144645553923, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8492144645553923
[LightGBM] [Warning] lambda_l2 is set=8.165710252546063, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.165710252546063
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8492144645553923, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8492144645553923
[LightGBM] [Warning] lambda_l2 is set=8.165710252546063, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.165710252546063
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8492144645553923, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8492144645553923
[LightGBM] [Warning] lambda_l2 is set=8.165710252546063, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.165710252546063
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[LightGBM] [Warning] lambda_l1 is set=2.8492144645553923, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.8492144645553923
[LightGBM] [Warning] lambda_l2 is set=8.165710252546063, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.165710252546063
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10


study.best_value

0.6886381294463726


def objective(trial):
    """Define the objective function"""

    params = {
        'max_depth': trial.suggest_int('max_depth', 1, 9),
        'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 1.0),
        'n_estimators': trial.suggest_int('n_estimators', 50, 500),
        'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),
        'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),
        'subsample': trial.suggest_loguniform('subsample', 0.01, 1.0),
        'colsample_bytree': trial.suggest_loguniform('colsample_bytree', 0.01, 1.0),
        'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 1.0),
        'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 1.0),
        'eval_metric': 'auc',
    }

    # Fit the model
    optuna_model = XGBClassifier(**params)
    return np.mean(cross_val_score(optuna_model, X, y, cv=skf, scoring='roc_auc'))


study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=20)

[I 2022-10-18 11:38:59,269] A new study created in memory with name: no-name-70e645e7-73f9-4380-9d4e-a3861587f553
[I 2022-10-18 11:39:15,400] Trial 0 finished with value: 0.5178008916210874 and parameters: {'max_depth': 6, 'learning_rate': 0.9847490216038848, 'n_estimators': 477, 'min_child_weight': 10, 'gamma': 0.18821671584654576, 'subsample': 0.017709621134238157, 'colsample_bytree': 0.05464365512300051, 'reg_alpha': 1.5657202160942697e-07, 'reg_lambda': 0.17937833815937104}. Best is trial 0 with value: 0.5178008916210874.
[I 2022-10-18 11:39:48,479] Trial 1 finished with value: 0.6612398526467234 and parameters: {'max_depth': 6, 'learning_rate': 0.25103068566507947, 'n_estimators': 278, 'min_child_weight': 6, 'gamma': 9.84128192105661e-07, 'subsample': 0.7314351958369125, 'colsample_bytree': 0.6306191865534457, 'reg_alpha': 3.991828023826425e-06, 'reg_lambda': 1.1153461072066687e-05}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:39:51,727] Trial 2 finished with value: 0.5990593528631399 and parameters: {'max_depth': 6, 'learning_rate': 0.03827346388240056, 'n_estimators': 60, 'min_child_weight': 7, 'gamma': 7.74429837346125e-05, 'subsample': 0.01930919569706319, 'colsample_bytree': 0.3388439116891192, 'reg_alpha': 4.630891212960109e-05, 'reg_lambda': 0.8112767987745361}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:40:07,047] Trial 3 finished with value: 0.6419780055018072 and parameters: {'max_depth': 1, 'learning_rate': 0.28881014454429016, 'n_estimators': 413, 'min_child_weight': 2, 'gamma': 0.00042585128589447507, 'subsample': 0.8223426188644173, 'colsample_bytree': 0.010208820571352164, 'reg_alpha': 0.0903601102222745, 'reg_lambda': 0.00019336530202732034}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:40:10,850] Trial 4 finished with value: 0.6283895771953552 and parameters: {'max_depth': 1, 'learning_rate': 0.081409110645201, 'n_estimators': 89, 'min_child_weight': 6, 'gamma': 0.005557502980415191, 'subsample': 0.20250095878084842, 'colsample_bytree': 0.8754646135563454, 'reg_alpha': 2.474608752436114e-08, 'reg_lambda': 0.05430958068368139}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:40:17,814] Trial 5 finished with value: 0.5891206692459316 and parameters: {'max_depth': 9, 'learning_rate': 0.8899459220198809, 'n_estimators': 84, 'min_child_weight': 1, 'gamma': 5.702187455046356e-08, 'subsample': 0.5269809660159449, 'colsample_bytree': 0.05726775253400361, 'reg_alpha': 0.10138215804709787, 'reg_lambda': 0.002795325518412142}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:40:26,222] Trial 6 finished with value: 0.6544533530128646 and parameters: {'max_depth': 5, 'learning_rate': 0.102321601366619, 'n_estimators': 129, 'min_child_weight': 10, 'gamma': 1.7683576777288827e-07, 'subsample': 0.201979996250591, 'colsample_bytree': 0.4312823076999437, 'reg_alpha': 0.018043829123886995, 'reg_lambda': 1.9869503084334187e-07}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:40:41,856] Trial 7 finished with value: 0.6412458169279182 and parameters: {'max_depth': 1, 'learning_rate': 0.1964699114980169, 'n_estimators': 402, 'min_child_weight': 2, 'gamma': 0.005928224586160517, 'subsample': 0.19857881835084615, 'colsample_bytree': 0.8131446819830787, 'reg_alpha': 0.006204789337660842, 'reg_lambda': 4.970815997187347e-06}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:41:03,332] Trial 8 finished with value: 0.6021807473611048 and parameters: {'max_depth': 7, 'learning_rate': 0.0819790492734755, 'n_estimators': 493, 'min_child_weight': 5, 'gamma': 4.576593169032308e-07, 'subsample': 0.01716524634551705, 'colsample_bytree': 0.09619909608772118, 'reg_alpha': 2.917828766375959e-07, 'reg_lambda': 0.0006742803840984148}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:41:19,449] Trial 9 finished with value: 0.6574118302914043 and parameters: {'max_depth': 7, 'learning_rate': 0.010058179437484176, 'n_estimators': 252, 'min_child_weight': 6, 'gamma': 1.1531844307491352e-07, 'subsample': 0.14489241080906876, 'colsample_bytree': 0.187548347486818, 'reg_alpha': 0.0025698296442670573, 'reg_lambda': 5.168754043135982e-08}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:41:29,461] Trial 10 finished with value: 0.5960521983857869 and parameters: {'max_depth': 4, 'learning_rate': 0.31463175967565193, 'n_estimators': 249, 'min_child_weight': 8, 'gamma': 1.0566432194790471e-05, 'subsample': 0.051328673278367536, 'colsample_bytree': 0.015956442986133683, 'reg_alpha': 7.696621311614376e-06, 'reg_lambda': 3.5207519627574413e-06}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:41:45,847] Trial 11 finished with value: 0.6510752156850524 and parameters: {'max_depth': 8, 'learning_rate': 0.010789786876001042, 'n_estimators': 264, 'min_child_weight': 4, 'gamma': 2.1201549844205974e-06, 'subsample': 0.07057608079134034, 'colsample_bytree': 0.2778573937185178, 'reg_alpha': 0.0008657176380372058, 'reg_lambda': 1.4183270067137154e-08}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:41:58,243] Trial 12 finished with value: 0.6609970847414344 and parameters: {'max_depth': 4, 'learning_rate': 0.011285665012928155, 'n_estimators': 198, 'min_child_weight': 4, 'gamma': 1.9313745928312004e-08, 'subsample': 0.44272987595232044, 'colsample_bytree': 0.17553297652429733, 'reg_alpha': 6.876706283625655e-06, 'reg_lambda': 4.9345075838917885e-06}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:42:09,029] Trial 13 finished with value: 0.6597928321462405 and parameters: {'max_depth': 3, 'learning_rate': 0.02934881962699279, 'n_estimators': 173, 'min_child_weight': 4, 'gamma': 1.8003996810940786e-08, 'subsample': 0.46432816895706447, 'colsample_bytree': 0.14718728876288084, 'reg_alpha': 1.5205060809970712e-05, 'reg_lambda': 5.739139801030376e-06}. Best is trial 1 with value: 0.6612398526467234.
[I 2022-10-18 11:42:37,070] Trial 14 finished with value: 0.6862289485954585 and parameters: {'max_depth': 3, 'learning_rate': 0.02631881316847292, 'n_estimators': 336, 'min_child_weight': 4, 'gamma': 1.0131566126567182e-08, 'subsample': 0.9646430480076363, 'colsample_bytree': 0.722218718616715, 'reg_alpha': 3.273715718793605e-06, 'reg_lambda': 3.965548535252624e-05}. Best is trial 14 with value: 0.6862289485954585.
[I 2022-10-18 11:43:05,140] Trial 15 finished with value: 0.6845960382276413 and parameters: {'max_depth': 3, 'learning_rate': 0.02464792616376107, 'n_estimators': 351, 'min_child_weight': 3, 'gamma': 2.724781510113401e-06, 'subsample': 0.8746563005909297, 'colsample_bytree': 0.5062718319817441, 'reg_alpha': 7.26294041746106e-07, 'reg_lambda': 6.739938550097768e-05}. Best is trial 14 with value: 0.6862289485954585.
[I 2022-10-18 11:43:27,614] Trial 16 finished with value: 0.6794706260576197 and parameters: {'max_depth': 3, 'learning_rate': 0.02561348388343415, 'n_estimators': 354, 'min_child_weight': 3, 'gamma': 1.5010709559835583e-05, 'subsample': 0.3385866854685991, 'colsample_bytree': 0.49290383702988505, 'reg_alpha': 9.097100486995425e-07, 'reg_lambda': 0.009077419624687784}. Best is trial 14 with value: 0.6862289485954585.
[I 2022-10-18 11:43:55,511] Trial 17 finished with value: 0.6867603909406768 and parameters: {'max_depth': 3, 'learning_rate': 0.019842835945463603, 'n_estimators': 342, 'min_child_weight': 3, 'gamma': 6.908021108956782e-06, 'subsample': 0.9098830935537209, 'colsample_bytree': 0.9985723966973875, 'reg_alpha': 0.00018406912928973625, 'reg_lambda': 0.00010144407751914216}. Best is trial 17 with value: 0.6867603909406768.
[I 2022-10-18 11:44:08,644] Trial 18 finished with value: 0.6341057287120382 and parameters: {'max_depth': 2, 'learning_rate': 0.04661776990751693, 'n_estimators': 327, 'min_child_weight': 1, 'gamma': 0.0003663235447394514, 'subsample': 0.03159408018214182, 'colsample_bytree': 0.8711452112999739, 'reg_alpha': 0.00034831506109148314, 'reg_lambda': 0.00015036298378285723}. Best is trial 17 with value: 0.6867603909406768.
[I 2022-10-18 11:44:27,979] Trial 19 finished with value: 0.6477454804276954 and parameters: {'max_depth': 4, 'learning_rate': 0.016320379772142894, 'n_estimators': 420, 'min_child_weight': 8, 'gamma': 0.2123624284680917, 'subsample': 0.2767432085429712, 'colsample_bytree': 0.028340793651780348, 'reg_alpha': 0.0001400554529589211, 'reg_lambda': 4.50988560749194e-07}. Best is trial 17 with value: 0.6867603909406768.


study.best_value

0.6867603909406768


study.best_params

{'max_depth': 3,
 'learning_rate': 0.019842835945463603,
 'n_estimators': 342,
 'min_child_weight': 3,
 'gamma': 6.908021108956782e-06,
 'subsample': 0.9098830935537209,
 'colsample_bytree': 0.9985723966973875,
 'reg_alpha': 0.00018406912928973625,
 'reg_lambda': 0.00010144407751914216}


lgbm_clf = LGBMClassifier(bagging_freq=10,
               class_weight={0: 1.0883478260869566, 1: 0.9249187112030742},
               colsample_bytree=0.6869647898646877, lambda_l1=9.543658512845797,
               lambda_l2=9.999550128682136, learning_rate=0.2637289335936835,
               max_bin=150, max_depth=4, metric='auc', min_child_samples=57,
               n_estimators=4000, num_leaves=67, random_state=752,
               subsample=0.8886905347670871)


# Flexible Stratified K-Fold Cross Validation

def lgb_predict(estimator, train, label, test, estimator_name): 
    mean_train = [] 
    mean_test_val = [] 
    test_pred = [] 
    val_pred = np.empty((len(train))) 
    for count, (train_index,test_index) in enumerate(skf.split(train,label)):
        x_train,x_test = train.iloc[train_index], train.iloc[test_index] 
        y_train,y_test = label.iloc[train_index], label.iloc[test_index] 
        
        print(f'========================Fold{count +1}==========================')
        
        
        estimator.fit(x_train, y_train, eval_set = [(x_train, y_train), (x_test, y_test)], verbose = 100,
                     early_stopping_rounds=200, eval_metric='auc'
                     )
                     
        
        train_predict = estimator.predict_proba(x_train)[:, 1]
        test_predict = estimator.predict_proba(x_test)[:, 1]
        val_pred[test_index] = (test_predict) # housing the lgbm-meta train data for stacking
        f_test= estimator.predict_proba(test)[:, 1]
        test_pred.append(f_test) # housing the lgbm-meta-test data for stacking

        print('\nValidation scores', roc_auc_score(y_test, test_predict))
        print('\nTraining scores', roc_auc_score(y_train, train_predict))
        mean_train.append(roc_auc_score(y_train, train_predict))
        mean_test_val.append(roc_auc_score(y_test,test_predict))
    print('Average Testing AUC_score score for 10 folds split:',np.mean(mean_test_val))
    print('Average Training AUC_score score for 10 folds split:',np.mean(mean_train))
    print('standard Deviation for 10 folds split:',np.std(mean_test_val))
    return val_pred, test_pred, estimator_name # return the meta train, meta-test, and estimator name


lgb_train, lgb_test, lgb_estimator = lgb_predict(lgbm_clf, X, y, test_copy, 'lgb_clf')

========================Fold1==========================
[LightGBM] [Warning] lambda_l1 is set=9.543658512845797, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.543658512845797
[LightGBM] [Warning] lambda_l2 is set=9.999550128682136, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.999550128682136
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[100]	training's auc: 0.795527	valid_1's auc: 0.698254
[200]	training's auc: 0.845132	valid_1's auc: 0.686905
[300]	training's auc: 0.875351	valid_1's auc: 0.686987

Validation scores 0.6995928328302614

Training scores 0.7975803043898755
========================Fold2==========================
[LightGBM] [Warning] lambda_l1 is set=9.543658512845797, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.543658512845797
[LightGBM] [Warning] lambda_l2 is set=9.999550128682136, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.999550128682136
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[100]	training's auc: 0.792974	valid_1's auc: 0.690114
[200]	training's auc: 0.845577	valid_1's auc: 0.687195

Validation scores 0.6949277503050544

Training scores 0.7813453420002651
========================Fold3==========================
[LightGBM] [Warning] lambda_l1 is set=9.543658512845797, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.543658512845797
[LightGBM] [Warning] lambda_l2 is set=9.999550128682136, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.999550128682136
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[100]	training's auc: 0.79339	valid_1's auc: 0.684645
[200]	training's auc: 0.840974	valid_1's auc: 0.678988
[300]	training's auc: 0.873862	valid_1's auc: 0.677018

Validation scores 0.6862475113993964

Training scores 0.7938954144759339
========================Fold4==========================
[LightGBM] [Warning] lambda_l1 is set=9.543658512845797, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.543658512845797
[LightGBM] [Warning] lambda_l2 is set=9.999550128682136, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.999550128682136
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[100]	training's auc: 0.796506	valid_1's auc: 0.684986
[200]	training's auc: 0.846051	valid_1's auc: 0.688467
[300]	training's auc: 0.874559	valid_1's auc: 0.685836

Validation scores 0.6889782287585897

Training scores 0.8449914671503137
========================Fold5==========================
[LightGBM] [Warning] lambda_l1 is set=9.543658512845797, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.543658512845797
[LightGBM] [Warning] lambda_l2 is set=9.999550128682136, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.999550128682136
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[100]	training's auc: 0.79433	valid_1's auc: 0.715263
[200]	training's auc: 0.84022	valid_1's auc: 0.715181
[300]	training's auc: 0.874294	valid_1's auc: 0.713521

Validation scores 0.7182043542482821

Training scores 0.812473272861571
========================Fold6==========================
[LightGBM] [Warning] lambda_l1 is set=9.543658512845797, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.543658512845797
[LightGBM] [Warning] lambda_l2 is set=9.999550128682136, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.999550128682136
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[100]	training's auc: 0.792621	valid_1's auc: 0.674143
[200]	training's auc: 0.840345	valid_1's auc: 0.681634
[300]	training's auc: 0.87485	valid_1's auc: 0.6761
[400]	training's auc: 0.896765	valid_1's auc: 0.668479

Validation scores 0.6826510821398755

Training scores 0.8419625157585985
========================Fold7==========================
[LightGBM] [Warning] lambda_l1 is set=9.543658512845797, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.543658512845797
[LightGBM] [Warning] lambda_l2 is set=9.999550128682136, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.999550128682136
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[100]	training's auc: 0.794444	valid_1's auc: 0.677067
[200]	training's auc: 0.845703	valid_1's auc: 0.679756
[300]	training's auc: 0.876694	valid_1's auc: 0.676743

Validation scores 0.6810470800102908

Training scores 0.838303419718712
========================Fold8==========================
[LightGBM] [Warning] lambda_l1 is set=9.543658512845797, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.543658512845797
[LightGBM] [Warning] lambda_l2 is set=9.999550128682136, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.999550128682136
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[100]	training's auc: 0.793997	valid_1's auc: 0.701716
[200]	training's auc: 0.842673	valid_1's auc: 0.700916

Validation scores 0.7087419603807563

Training scores 0.7683402108469575
========================Fold9==========================
[LightGBM] [Warning] lambda_l1 is set=9.543658512845797, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.543658512845797
[LightGBM] [Warning] lambda_l2 is set=9.999550128682136, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.999550128682136
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[100]	training's auc: 0.789256	valid_1's auc: 0.699359
[200]	training's auc: 0.841032	valid_1's auc: 0.689789

Validation scores 0.7080061744275791

Training scores 0.7596681659170414
========================Fold10==========================
[LightGBM] [Warning] lambda_l1 is set=9.543658512845797, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.543658512845797
[LightGBM] [Warning] lambda_l2 is set=9.999550128682136, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.999550128682136
[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10
[100]	training's auc: 0.792941	valid_1's auc: 0.68353
[200]	training's auc: 0.843641	valid_1's auc: 0.680286
[300]	training's auc: 0.875357	valid_1's auc: 0.679892

Validation scores 0.684998713660921

Training scores 0.8151549622014389
Average Testing AUC_score score for 10 folds split: 0.6953395688161007
Average Training AUC_score score for 10 folds split: 0.8053715075320707
standard Deviation for 10 folds split: 0.012150981979339247


lgb_test_ = np.mean(lgb_test, axis=0)


plt.figure(figsize=(12,8))
feat_importances = pd.Series(lgbm_clf.feature_importances_, index=X.columns)
feat_importances.sort_values().plot(kind='barh',title='lightgbm_1 feature importance');


xgb_clf = XGBClassifier(colsample_bytree=0.6682960151866543, gamma=1.0798626152988356e-08,
              learning_rate=0.02320865571556698, max_depth=6,
              min_child_weight=3, n_estimators=196,
              reg_alpha=0.060016599138875734, reg_lambda=3.429258701297801e-05,
              subsample=0.5054109001304139)


def xgb_predict(estimator, train, label, test, estimator_name):
    mean_train = []
    mean_test_val = []
    test_pred = []
    val_pred = np.empty((len(train)))
    for count, (train_index,test_index) in enumerate(skf.split(train,label)):
        x_train,x_test = train.iloc[train_index], train.iloc[test_index]
        y_train,y_test = label.iloc[train_index], label.iloc[test_index]
        
        print(f'========================Fold{count +1}==========================')
        
        
        estimator.fit(x_train, y_train, eval_set = [(x_train, y_train), (x_test, y_test)], verbose = 100,
                     early_stopping_rounds=200 
                     )
                     
        
        train_predict = estimator.predict_proba(x_train)[:, 1]
        test_predict = estimator.predict_proba(x_test)[:, 1]
        val_pred[test_index] = (test_predict)
        f_test= estimator.predict_proba(test)[:, 1]
        test_pred.append(f_test)
    
        print('\nValidation scores', roc_auc_score(y_test, test_predict))
        print('\nTraining scores', roc_auc_score(y_train, train_predict))
        mean_train.append(roc_auc_score(y_train, train_predict))
        mean_test_val.append(roc_auc_score(y_test,test_predict))
    print('Average Testing AUC_score score for 10 folds split:',np.mean(mean_test_val))
    print('Average Training AUC_score score for 10 folds split:',np.mean(mean_train))
    print('standard Deviation for 10 folds split:',np.std(mean_test_val))
    return val_pred, test_pred, estimator_name


xgb_train, xgb_test, xgb_estimator = xgb_predict(xgb_clf, X, y, test_copy, 'xgb_clf')

========================Fold1==========================
[0]	validation_0-logloss:0.69099	validation_1-logloss:0.69143
[100]	validation_0-logloss:0.60411	validation_1-logloss:0.64130
[195]	validation_0-logloss:0.56846	validation_1-logloss:0.63094

Validation scores 0.6972628604457003

Training scores 0.8210335895171701
========================Fold2==========================
[0]	validation_0-logloss:0.69115	validation_1-logloss:0.69138
[100]	validation_0-logloss:0.60376	validation_1-logloss:0.63566
[195]	validation_0-logloss:0.56676	validation_1-logloss:0.62794

Validation scores 0.692466765140325

Training scores 0.8253271798435922
========================Fold3==========================
[0]	validation_0-logloss:0.69119	validation_1-logloss:0.69152
[100]	validation_0-logloss:0.60084	validation_1-logloss:0.63842
[195]	validation_0-logloss:0.56558	validation_1-logloss:0.62824

Validation scores 0.6855898786205125

Training scores 0.8228674817898436
========================Fold4==========================
[0]	validation_0-logloss:0.69104	validation_1-logloss:0.69203
[100]	validation_0-logloss:0.60212	validation_1-logloss:0.64569
[195]	validation_0-logloss:0.56599	validation_1-logloss:0.63963

Validation scores 0.6771125810802133

Training scores 0.8221102598096036
========================Fold5==========================
[0]	validation_0-logloss:0.69103	validation_1-logloss:0.69080
[100]	validation_0-logloss:0.60373	validation_1-logloss:0.62277
[195]	validation_0-logloss:0.56818	validation_1-logloss:0.60859

Validation scores 0.7266713762764113

Training scores 0.8254295264367598
========================Fold6==========================
[0]	validation_0-logloss:0.69107	validation_1-logloss:0.69173
[100]	validation_0-logloss:0.60284	validation_1-logloss:0.64220
[195]	validation_0-logloss:0.56671	validation_1-logloss:0.63186

Validation scores 0.6902061524629118

Training scores 0.8252930801802252
========================Fold7==========================
[0]	validation_0-logloss:0.69114	validation_1-logloss:0.69131
[100]	validation_0-logloss:0.60131	validation_1-logloss:0.63836
[195]	validation_0-logloss:0.56792	validation_1-logloss:0.63035

Validation scores 0.6853820427064574

Training scores 0.8188160681563981
========================Fold8==========================
[0]	validation_0-logloss:0.69113	validation_1-logloss:0.69127
[100]	validation_0-logloss:0.60195	validation_1-logloss:0.62701
[195]	validation_0-logloss:0.56602	validation_1-logloss:0.62005

Validation scores 0.7012271674813481

Training scores 0.8245020981572706
========================Fold9==========================
[0]	validation_0-logloss:0.69116	validation_1-logloss:0.69120
[100]	validation_0-logloss:0.60123	validation_1-logloss:0.62928
[195]	validation_0-logloss:0.56833	validation_1-logloss:0.62097

Validation scores 0.7085644455878569

Training scores 0.8189070068140534
========================Fold10==========================
[0]	validation_0-logloss:0.69102	validation_1-logloss:0.69134
[100]	validation_0-logloss:0.60055	validation_1-logloss:0.63466
[195]	validation_0-logloss:0.56642	validation_1-logloss:0.63097

Validation scores 0.693293028042192

Training scores 0.8086236246955887
Average Testing AUC_score score for 10 folds split: 0.695777629784393
Average Training AUC_score score for 10 folds split: 0.8212909915400506
standard Deviation for 10 folds split: 0.013259896604372852


xgb_test_ = np.mean(xgb_test, axis=0)


xgb_2 = XGBClassifier()
xgb_2.set_params(**{'max_depth': 8,
 'learning_rate': 0.03097330142137865,
 'n_estimators': 499,
 'min_child_weight': 3,
 'gamma': 0.00014108851650059825,
 'subsample': 0.9231145138691651,
 'colsample_bytree': 0.9670536338693626,
 'reg_alpha': 0.00011310709950064508,
 'reg_lambda': 1.2597877821986132e-08})

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=0.9670536338693626, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None,
              gamma=0.00014108851650059825, gpu_id=None, grow_policy=None,
              importance_type=None, interaction_constraints=None,
              learning_rate=0.03097330142137865, max_bin=None,
              max_cat_to_onehot=None, max_delta_step=None, max_depth=8,
              max_leaves=None, min_child_weight=3, missing=nan,
              monotone_constraints=None, n_estimators=499, n_jobs=None,
              num_parallel_tree=None, predictor=None, random_state=None,
              reg_alpha=0.00011310709950064508,
              reg_lambda=1.2597877821986132e-08, ...)

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=0.9670536338693626, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None,
              gamma=0.00014108851650059825, gpu_id=None, grow_policy=None,
              importance_type=None, interaction_constraints=None,
              learning_rate=0.03097330142137865, max_bin=None,
              max_cat_to_onehot=None, max_delta_step=None, max_depth=8,
              max_leaves=None, min_child_weight=3, missing=nan,
              monotone_constraints=None, n_estimators=499, n_jobs=None,
              num_parallel_tree=None, predictor=None, random_state=None,
              reg_alpha=0.00011310709950064508,
              reg_lambda=1.2597877821986132e-08, ...)


xgb_2_train, xgb_2_test, xgb_2_estimator = xgb_predict(xgb_2, X, y, test_copy, 'xgb_clf_2')

========================Fold1==========================
[0]	validation_0-logloss:0.68923	validation_1-logloss:0.69038
[100]	validation_0-logloss:0.52919	validation_1-logloss:0.62294
[200]	validation_0-logloss:0.45887	validation_1-logloss:0.61575
[300]	validation_0-logloss:0.40953	validation_1-logloss:0.61404
[400]	validation_0-logloss:0.36457	validation_1-logloss:0.61301
[498]	validation_0-logloss:0.32806	validation_1-logloss:0.61516

Validation scores 0.7182762828334724

Training scores 0.9888849219666731
========================Fold2==========================
[0]	validation_0-logloss:0.68897	validation_1-logloss:0.69029
[100]	validation_0-logloss:0.52761	validation_1-logloss:0.61902
[200]	validation_0-logloss:0.46309	validation_1-logloss:0.61631
[300]	validation_0-logloss:0.41278	validation_1-logloss:0.61918
[400]	validation_0-logloss:0.36929	validation_1-logloss:0.62171

Validation scores 0.7003789095112709

Training scores 0.9385252728647446
========================Fold3==========================
[0]	validation_0-logloss:0.68886	validation_1-logloss:0.69067
[100]	validation_0-logloss:0.52480	validation_1-logloss:0.62546
[200]	validation_0-logloss:0.46502	validation_1-logloss:0.62169
[300]	validation_0-logloss:0.41325	validation_1-logloss:0.62093
[400]	validation_0-logloss:0.36823	validation_1-logloss:0.62414
[443]	validation_0-logloss:0.35009	validation_1-logloss:0.62611

Validation scores 0.6952668422066662

Training scores 0.9496416044454916
========================Fold4==========================
[0]	validation_0-logloss:0.68909	validation_1-logloss:0.69113
[100]	validation_0-logloss:0.52784	validation_1-logloss:0.63756
[200]	validation_0-logloss:0.46635	validation_1-logloss:0.63621
[300]	validation_0-logloss:0.41508	validation_1-logloss:0.63527
[400]	validation_0-logloss:0.37123	validation_1-logloss:0.63540
[498]	validation_0-logloss:0.33632	validation_1-logloss:0.63798

Validation scores 0.6864632971549676

Training scores 0.9762135092742673
========================Fold5==========================
[0]	validation_0-logloss:0.68892	validation_1-logloss:0.68946
[100]	validation_0-logloss:0.53275	validation_1-logloss:0.60256
[200]	validation_0-logloss:0.46860	validation_1-logloss:0.59920
[300]	validation_0-logloss:0.41486	validation_1-logloss:0.59874
[400]	validation_0-logloss:0.37130	validation_1-logloss:0.60027
[442]	validation_0-logloss:0.35379	validation_1-logloss:0.60001

Validation scores 0.7308971806563485

Training scores 0.9517173837671958
========================Fold6==========================
[0]	validation_0-logloss:0.68914	validation_1-logloss:0.69100
[100]	validation_0-logloss:0.52565	validation_1-logloss:0.62840
[200]	validation_0-logloss:0.46306	validation_1-logloss:0.62679
[300]	validation_0-logloss:0.41116	validation_1-logloss:0.62849
[336]	validation_0-logloss:0.39669	validation_1-logloss:0.62813

Validation scores 0.6945655385010598

Training scores 0.9027327333760173
========================Fold7==========================
[0]	validation_0-logloss:0.68900	validation_1-logloss:0.69063
[100]	validation_0-logloss:0.52805	validation_1-logloss:0.62535
[200]	validation_0-logloss:0.46855	validation_1-logloss:0.62328
[300]	validation_0-logloss:0.41605	validation_1-logloss:0.62643
[367]	validation_0-logloss:0.38696	validation_1-logloss:0.62789

Validation scores 0.6935914587085156

Training scores 0.9180671093024917
========================Fold8==========================
[0]	validation_0-logloss:0.68937	validation_1-logloss:0.69025
[100]	validation_0-logloss:0.53095	validation_1-logloss:0.60981
[200]	validation_0-logloss:0.46755	validation_1-logloss:0.60668
[300]	validation_0-logloss:0.41416	validation_1-logloss:0.60672
[400]	validation_0-logloss:0.37248	validation_1-logloss:0.60449
[498]	validation_0-logloss:0.33494	validation_1-logloss:0.60663

Validation scores 0.7174324671983534

Training scores 0.9863795086583692
========================Fold9==========================
[0]	validation_0-logloss:0.68918	validation_1-logloss:0.69045
[100]	validation_0-logloss:0.52991	validation_1-logloss:0.61273
[200]	validation_0-logloss:0.47035	validation_1-logloss:0.60818
[300]	validation_0-logloss:0.41911	validation_1-logloss:0.60772
[400]	validation_0-logloss:0.37545	validation_1-logloss:0.60829
[498]	validation_0-logloss:0.33843	validation_1-logloss:0.60950

Validation scores 0.7221456135837406

Training scores 0.9690082260457074
========================Fold10==========================
[0]	validation_0-logloss:0.68910	validation_1-logloss:0.69005
[100]	validation_0-logloss:0.53147	validation_1-logloss:0.62495
[200]	validation_0-logloss:0.46190	validation_1-logloss:0.62515
[300]	validation_0-logloss:0.41396	validation_1-logloss:0.62844
[383]	validation_0-logloss:0.37733	validation_1-logloss:0.63279

Validation scores 0.6997478775405197

Training scores 0.9300107565264988
Average Testing AUC_score score for 10 folds split: 0.7058765467894915
Average Training AUC_score score for 10 folds split: 0.9511181026227458
standard Deviation for 10 folds split: 0.01419187935645962


xgb_2_test = np.mean(xgb_2_test, axis=0)


# preparing model results for ensembling:
xgb_train = pd.DataFrame(xgb_train)
xgb_2_train = pd.DataFrame(xgb_2_train)
lgb_train = pd.DataFrame(lgb_train)
# ---------------------------------------
stack_train = pd.concat((xgb_train, xgb_2_train, lgb_train), axis=1)
stack_train.head()


xgb_test_ = pd.DataFrame(xgb_test_)
xgb_2_test = pd.DataFrame(xgb_2_test)
lgb_test_ = pd.DataFrame(lgb_test_)
# -----------------------------------
stack_test = pd.concat((xgb_test_, xgb_2_test, lgb_test_), axis=1)
stack_test.head()


# using Linear Regression as the first meta model 
lr = LinearRegression()
lr.fit(stack_train, y)

pred = lr.predict(stack_train)
pred

array([0.62509204, 0.36670404, 0.70218927, ..., 0.44126959, 0.53779044,
       0.65739997])


print(roc_auc_score(y, pred))

0.7081958128237094


# using Logistic Regression as the second meta model:
lg = LogisticRegression()
lg.fit(stack_train,y)

lg_pred = lg.predict_proba(stack_train)


print(roc_auc_score(y, lg_pred[:,1]))

0.708093948000874


# on stack test;

pred = lr.predict(stack_test)

pred_lg = lg.predict_proba(stack_test)[:,1]


stack_submission_dict = {
    'Entry_id': test_entry_Id,
    'e_signed' : pred_lg #logisticRegression meta model performed better on LB
}


stack_submission = pd.DataFrame(stack_submission_dict)
#stack_submission.to_csv('stacked_models_lg.csv',index=False)

	Entry_id	age	pay_schedule	home_owner	income	months_employed	years_employed	current_address_year	personal_account_m	personal_account_y	has_debt	amount_requested	risk_score	risk_score_2	risk_score_3	risk_score_4	risk_score_5	ext_quality_score	ext_quality_score_2	inquiries_last_month	e_signed
2488	3903643	41	bi-weekly	0	3035	0	5	0	2	6	0	700	56550	0.574753	0.843991	0.241663	0.516921	0.715925	0.715925	15	1
9804	3903643	38	bi-weekly	1	2620	0	2	0	6	4	1	400	67050	0.740419	0.939486	0.684477	0.778170	0.629805	0.729805	9	1
7213	5659209	42	bi-weekly	1	2140	0	2	9	6	4	1	400	56700	0.781691	0.920329	0.481009	0.594963	0.736846	0.536846	6	1
7288	5659209	46	bi-weekly	1	7092	0	3	1	2	3	0	700	64350	0.679462	0.757643	0.550134	0.391500	0.487792	0.587792	6	1
5361	5896278	31	weekly	0	1635	0	6	5	2	2	1	400	73200	0.797840	0.849569	0.687969	0.741433	0.696454	0.696454	6	0
11829	5896278	30	bi-weekly	0	3795	0	2	3	1	3	1	800	50650	0.760728	0.850893	0.573459	0.606560	0.564857	0.564857	4	1
4419	5946902	43	semi-monthly	1	4725	3	6	6	5	8	1	1000	59250	0.724398	0.944054	0.444145	0.582691	0.698231	0.698231	3	1
12124	5946902	41	semi-monthly	1	4295	2	2	1	3	2	1	1200	54100	0.681107	0.860848	0.525299	0.689527	0.421167	0.621167	9	0
2626	7450902	57	bi-weekly	0	5295	0	6	4	2	6	1	1700	91500	0.648824	0.789179	0.640210	0.653759	0.694329	0.594329	5	0
9209	7450902	35	bi-weekly	0	4095	6	2	0	1	3	0	700	57850	0.780735	0.929274	0.625784	0.792078	0.550449	0.550449	5	0
1487	8156839	37	bi-weekly	0	3020	0	1	7	4	4	1	600	53550	0.728198	0.881015	0.647649	0.742549	0.594076	0.494076	5	1
11067	8156839	48	bi-weekly	0	3205	0	2	0	5	3	1	500	53650	0.623579	0.801490	0.621682	0.654174	0.395154	0.495154	4	1

	Entry_id	age	home_owner	income	months_employed	years_employed	current_address_year	personal_account_m	personal_account_y	has_debt	amount_requested	risk_score	risk_score_2	risk_score_3	risk_score_4	risk_score_5	ext_quality_score	ext_quality_score_2	inquiries_last_month	e_signed
count	1.251600e+04	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000	12516.000000
mean	5.599485e+06	43.026766	0.425056	3654.110658	1.191355	3.532998	3.593161	3.402205	3.508469	0.795462	950.251518	61133.401246	0.690665	0.877954	0.582848	0.718456	0.621944	0.621912	6.494088	0.540588
std	2.563354e+06	11.865891	0.494371	1508.072163	2.404529	2.265293	2.749211	2.206649	1.959969	0.403380	706.915146	15396.809340	0.090294	0.054197	0.125260	0.121103	0.139876	0.139623	3.731870	0.498370
min	1.111398e+06	18.000000	0.000000	905.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	350.000000	2100.000000	0.023258	0.451371	0.016724	0.153367	0.010184	0.006622	1.000000	0.000000
25%	3.376885e+06	34.000000	0.000000	2575.000000	0.000000	2.000000	2.000000	2.000000	2.000000	1.000000	600.000000	49450.000000	0.641221	0.850893	0.499887	0.635005	0.519794	0.519662	4.000000	0.000000
50%	5.608376e+06	42.000000	0.000000	3255.000000	0.000000	3.000000	3.000000	2.000000	3.000000	1.000000	700.000000	61200.000000	0.699621	0.880993	0.588202	0.725717	0.624081	0.623781	6.000000	1.000000
75%	7.803325e+06	51.000000	1.000000	4661.250000	1.000000	5.000000	5.000000	5.000000	4.000000	1.000000	1100.000000	72750.000000	0.752053	0.911485	0.672431	0.806576	0.729609	0.728389	8.000000	1.000000
max	9.999874e+06	96.000000	1.000000	9985.000000	11.000000	16.000000	12.000000	11.000000	14.000000	1.000000	10200.000000	99750.000000	0.999997	0.999012	0.978932	0.993363	0.970249	0.966953	30.000000	1.000000

	0	0	0
0	0.576665	0.565816	0.737682
1	0.427056	0.427855	0.206949
2	0.700967	0.700313	0.650142
3	0.585420	0.558738	0.454794
4	0.556723	0.482709	0.451343

	0	0	0
0	0.360520	0.308466	0.333925
1	0.720678	0.840729	0.776093
2	0.530376	0.587887	0.517935
3	0.571721	0.576584	0.611566
4	0.393138	0.332143	0.255641

Team Nova DataFest22's Datathon ML Submission¶

Project Overview¶

Background¶

Project Task¶

Importing the Packages¶

Data Gathering¶

Columns¶

Data Assessing and Cleaning¶

Explorative Data Analysis:¶

Percentage of clients that are e-signed¶

Percentage of Clients that are E-signed and in debt¶

Percentage of Clients that are E-signed and house owners¶

Age Distribution of the Clients with respect to E-signed¶

Examining the Number of clients and their pay schedule¶

Examining the distribution of amount requested¶

To Examine the relationship between income and amount requested¶

Examining the relationship between risk score and external quality score and Income¶

To Examine the relationship between number of enquiries last month and E-signed¶

Conclusion:¶

Model Development:¶

Feature Engineering;¶

Model Building:¶

Attempting a Narrower optimization:¶

Model Ensembling:¶

preparing for submission:`¶

	Entry_id	age	pay_schedule	home_owner	income	years_employed	current_address_year	personal_account_m	personal_account_y	has_debt	amount_requested	risk_score	risk_score_2	risk_score_3	risk_score_4	risk_score_5	ext_quality_score	ext_quality_score_2	inquiries_last_month	e_signed
0	7629673	40	bi-weekly	1	3135	3	3	6	2	1	550	36200	0.737398	0.903517	0.487712	0.515977	0.580918	0.380918	10	1
1	5335819	33	semi-monthly	0	3590	5	2	2	8	1	1100	53850	0.617361	0.857560	0.613487	0.665523	0.744634	0.744634	12	0
2	8492423	21	weekly	0	2303	5	8	2	7	1	600	74850	0.677109	0.758765	0.495609	0.664762	0.592556	0.492556	6	1
3	7948313	26	bi-weekly	0	2795	4	4	1	6	1	800	50800	0.738055	0.873204	0.666437	0.700392	0.584130	0.684130	14	1
4	4297036	43	bi-weekly	0	5000	2	1	1	2	1	1100	69100	0.798303	0.841747	0.401971	0.568787	0.525905	0.725905	5	1